from langchain_community.document_loaders import UnstructuredFileLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter, Language

# 构建Markdown 文档加载器并获取文档列表
# LLMOps 项目 API 文档
loader = UnstructuredFileLoader("./study_blob_example.py")
documents = loader.load()

# 构建分割器
text_splitter = RecursiveCharacterTextSplitter.from_language(
    Language.PYTHON,
    chunk_size=500,
    chunk_overlap=50,
    add_start_index=True
)

# 分割文档列表
chunks = text_splitter.split_documents(documents)

# 输出信息
for chunk in chunks:
    print("======================")
    print(f"块大小{len(chunk.page_content)},元数据{chunk.metadata}")
    print(chunk.page_content)
    print("==================")